#include <fermat/common/stream_tokenizer.h>
#include <fermat/common/token.h>


namespace fermat {


StreamTokenizer::StreamTokenizer():
	_pIstr(0)
{
}


StreamTokenizer::StreamTokenizer(std::istream& istr):
	_pIstr(&istr)
{
}


StreamTokenizer::~StreamTokenizer()
{
	for (TokenVec::iterator it = _tokens.begin(); it != _tokens.end(); ++it) {
		delete *it;
	}
}


void StreamTokenizer::attachToStream(std::istream& istr)
{
	_pIstr = &istr;
}


void StreamTokenizer::addToken(Token* pToken)
{

	pToken->ignore(pToken->tokenClass() == Token::COMMENT_TOKEN || pToken->tokenClass() == Token::WHITESPACE_TOKEN);
	_tokens.push_back(pToken);
}


void StreamTokenizer::addToken(Token* pToken, bool ignore)
{

	pToken->ignore(ignore);
	_tokens.push_back(pToken);
}

	
const Token* StreamTokenizer::next()
{
	
	static const int eof = std::char_traits<char>::eof();

	int first = _pIstr->get();
	TokenVec::const_iterator it = _tokens.begin();
	while (first != eof && it != _tokens.end()) {
		Token* ti = *it;
		if (ti->start((char) first, *_pIstr)) {
			ti->finish(*_pIstr);
			if (ti->ignored())  {
				first = _pIstr->get();
				it = _tokens.begin();
			} else {
				return *it;
			}
		} else {
			++it;
		}
	}
	if (first == eof) {
		return &_eofToken;
	} else {
		_invalidToken.start((char) first, *_pIstr);
		return &_invalidToken;
	}
}


} 